//package org.jeecg.modules.htxincailiao.crawler.processor;
//
//import com.alibaba.fastjson.JSONObject;
//import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
//import com.baomidou.mybatisplus.core.metadata.IPage;
//import lombok.extern.slf4j.Slf4j;
//import org.jeecg.common.system.query.QueryGenerator;
//import org.jeecg.modules.htxincailiao.news.entity.HTNewsCategory;
//import org.jeecg.modules.htxincailiao.news.service.IHTNewsCategoryService;
//import org.jeecg.modules.htxincailiao.news.service.IHTNewsService;
//import org.springframework.stereotype.Component;
//import us.codecraft.webmagic.Page;
//import us.codecraft.webmagic.Request;
//import us.codecraft.webmagic.Site;
//import us.codecraft.webmagic.Spider;
//import us.codecraft.webmagic.processor.PageProcessor;
//import us.codecraft.webmagic.selector.Html;
//
//import javax.annotation.Resource;
//import java.util.*;
//
//
///**
// * @author code4crafter@gmail.com <br>
// */
//@Component
//@Slf4j
//public class XinCaiLiaoOnlinePageProcessor implements PageProcessor {
////    public static final String URL_LIST = "http://www.xincailiao.com/news/news_list.aspx?fenlei=latest";
//
////    https://xincailiao.ofweek.com/CATList-180400-8100-xincailiao.html
//    public static final String URL_LIST = "https://xincailiao.ofweek.com/CAT-180422-inorganic.html";
//
//    public static final String URL_POST = "/news/news_detail.aspx?id=";
//    public static final String URL_BASE = "http://www.xincailiao.com/";
//
//    private static final String targetSite="xincailiao.com";
//
//    @Resource
//    private IHTNewsService newsService;
//
//    @Resource
//    private IHTNewsCategoryService newsCategoryService;
//
//
//    private Site site = Site
//            .me()
//            .setDomain(targetSite)
//            .setSleepTime(3000)
//            .setUserAgent(
//                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
//
//
//    @Override
//    public void process(Page page) {
//        //列表页
//        if (page.getRequest().getUrl().equals(URL_LIST)){
//            String liSrcXPath = "//div[@class='mui-h-news-HotList mui-inside-list']/ul/li/html()";
//
//            List<String> liList = page.getHtml().xpath(liSrcXPath).all();
//            for (String liString: liList) {
//                Html liHtml = new Html(liString);
//
//                String href = liHtml.xpath("//a/@href").get();
//                String cover = liHtml.xpath("//a/img/@src").get();
//                String subname = liHtml.xpath("//div[@class='mui-h-news-con']/span[@class='tabloid']/text()").get();
//
//                Map<String, Object> extras = new HashMap<>( );
//                extras.put("cover", cover);
//                extras.put("subname", subname);
//
//                String requestURL = URL_BASE + href;
//                Request request = new Request( );
//                request.setUrl(requestURL);
//                request.setExtras(extras);
//
//                page.addTargetRequest(request);
//            }
//
//            //文章详情页
//        } else {
//            String nameXPath = "//h1/text()";
//            String contentXPath = "//div[@class='newsDetail']/html()";
//            String sourceXPath = "//div[@class='time-source']/p/b/text()";
//
//            page.putField("name", page.getHtml().xpath(nameXPath));
//            page.putField("content", page.getHtml().xpath(contentXPath));
//            page.putField("source", page.getHtml().xpath(sourceXPath));
//
//            List<HTNewsCategory> categoryList = newsCategoryService.findAll();
//           if (!categoryList.isEmpty()) {
//               HTNewsCategory category = categoryList.get(0);
//               String categoryId = category.getId();
//
//               newsService.saveForCrawlerCollectBy(page,categoryId,nameXPath, sourceXPath, targetSite, contentXPath );
//           }
//
//        }
//
//    }
//
//    @Override
//    public Site getSite() {
//        return site;
//    }
//
//    public static void create(XinCaiLiaoOnlinePageProcessor pageProcessor){
//        Spider.create(pageProcessor).addUrl(URL_LIST).run();
//    }
//
////    public static void main(String[] args) {
//////        ConfigurableApplicationContext application = SpringApplication.run(JeecgSystemApplication.class, args);
//////        XinCaiLiaoOnlinePageProcessor pageProcessor =  new XinCaiLiaoOnlinePageProcessor();
//////        pageProcessor.newsService = (IHTNewsService)application.getBean("HTNewsServiceImpl");
//////
//////        Spider.create(pageProcessor).addUrl(URL_LIST)
//////                .run();
////    }
//}
